{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 卷积神经网络\n",
    "\n",
    "## 全连接网络的问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                multiple                  200960    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              multiple                  65792     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              multiple                  65792     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              multiple                  2570      \n",
      "=================================================================\n",
      "Total params: 335,114\n",
      "Trainable params: 335,114\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers,Sequential,losses,optimizers,datasets\n",
    "\n",
    "# 获取所有 GPU 设备列表\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU0')\n",
    "if gpus:\n",
    "    try:\n",
    "        # 设置 GPU 为增长式占用\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    except RuntimeError as e:\n",
    "        # 打印异常\n",
    "        print(e)\n",
    "\n",
    "# 创建 4 层全连接网络\n",
    "model = keras.Sequential([\n",
    "    layers.Dense(256, activation='relu'),\n",
    "    layers.Dense(256, activation='relu'),\n",
    "    layers.Dense(256, activation='relu'),\n",
    "    layers.Dense(10),\n",
    "])\n",
    "# build 模型，并打印模型信息\n",
    "model.build(input_shape=(4, 784))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**卷积运算：**  \n",
    "在信号处理领域，1D连续信号的卷积运算被定义两个函数的积分：函数$f(\\tau)$、函数$g(\\tau)$，其中$g(\\tau)$经过了翻转$g(-\\tau)$和平移后变成$g(n-\\tau)$。卷积的“卷”是指翻转平移操作，“积”是指积分运算，1D连续卷积定义为：$$(f\\otimes g)(n)=\\int_{-\\infty}^{\\infty}f(\\tau)g(n-\\tau)d\\tau$$离散卷积将积分运算换成累加运算：$$(f\\otimes g)(n)=\\sum_{\\tau=-\\infty}^{\\infty}f(\\tau)g(n-\\tau)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "卷积神经层的输出尺寸$[b,h',w',c_{out}]$由卷积核的数量$c_{out}$，卷积核的大小$k$，步长$s$，填充数$p$（只考虑上下填充数量$p_h$相同，左右填充数量$p_w$相同的情况）以及输入$X$的高宽$h/w$共同决定， 它们之间的数学关系可以表达为：$$h'=\\left\\lfloor \\frac{h+2 \\cdot p_h - k}{s} \\right\\rfloor + 1 \\\\ w'=\\left\\lfloor\\frac{w+2 \\cdot p_w - k}{s} \\right\\rfloor + 1$$其中$p_h$、$p_w$分别表示高、宽方向的填充数量，$\\lfloor \\cdot \\rfloor$表示向下取整。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 卷积层实现\n",
    "\n",
    "### 自定义权值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3, 3, 4])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模拟输入， 3 通道，高宽为 5\n",
    "x = tf.random.normal([2,5,5,3]) \n",
    "# 需要根据[k,k,cin,cout]格式创建 W 张量， 4 个 3x3 大小卷积核\n",
    "w = tf.random.normal([3,3,3,4])\n",
    "# 步长为 1, padding 为 0,\n",
    "out = tf.nn.conv2d(x,w,strides=1,padding=[[0,0],[0,0],[0,0],[0,0]])\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上下左右各填充一个单位，则 padding 参数设置为[[0,0],[1,1],[1,1],[0,0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 5, 5, 4])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.random.normal([2,5,5,3]) # 模拟输入， 3 通道，高宽为 5\n",
    "# 需要根据[k,k,cin,cout]格式创建， 4 个 3x3 大小卷积核\n",
    "w = tf.random.normal([3,3,3,4])\n",
    "# 步长为 1, padding 为 1,\n",
    "out = tf.nn.conv2d(x,w,strides=1,padding=[[0,0],[1,1],[1,1],[0,0]])\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 5, 5, 4])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.random.normal([2,5,5,3]) # 模拟输入， 3 通道，高宽为 5\n",
    "w = tf.random.normal([3,3,3,4]) # 4 个 3x3 大小的卷积核\n",
    "# 步长为,padding 设置为输出、输入同大小\n",
    "# 需要注意的是, padding=same 只有在 strides=1 时才是同大小\n",
    "out = tf.nn.conv2d(x,w,strides=1,padding='SAME')\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 2, 2, 4])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.random.normal([2,5,5,3])\n",
    "w = tf.random.normal([3,3,3,4])\n",
    "# 高宽先 padding 成可以整除 3 的最小整数 6，然后 6 按 3 倍减少，得到 2x2\n",
    "out = tf.nn.conv2d(x,w,strides=3,padding='SAME')\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 2, 2, 4])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 根据[cout]格式创建偏置向量\n",
    "b = tf.zeros([4])\n",
    "# 在卷积输出上叠加偏置向量，它会自动 broadcasting 为[b,h',w',cout]\n",
    "out = out + b\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 卷积层类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 5, 5, 4])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer = layers.Conv2D(4,kernel_size=3,strides=1,padding='SAME')\n",
    "out = layer(x) # 前向计算\n",
    "out.shape # 输出张量的 shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'conv2d/kernel:0' shape=(3, 3, 3, 4) dtype=float32, numpy=\n",
       " array([[[[-1.65961877e-01, -1.66751057e-01, -1.99422598e-01,\n",
       "            2.48859316e-01],\n",
       "          [ 2.05890387e-01,  1.17410600e-01, -2.71057725e-01,\n",
       "            4.53480780e-02],\n",
       "          [-8.31105113e-02, -2.25279003e-01,  9.72283483e-02,\n",
       "           -2.35820100e-01]],\n",
       " \n",
       "         [[-1.61854327e-01,  2.28703648e-01,  1.32889420e-01,\n",
       "           -3.55903506e-02],\n",
       "          [-2.19753847e-01, -9.75874811e-02,  1.67854369e-01,\n",
       "           -1.31139144e-01],\n",
       "          [-7.00651258e-02,  3.06583852e-01, -4.95633483e-03,\n",
       "            1.48773402e-01]],\n",
       " \n",
       "         [[-6.97237998e-02,  5.67982793e-02,  2.96313435e-01,\n",
       "           -2.80013651e-01],\n",
       "          [ 3.05833727e-01,  1.73607528e-01,  1.84971690e-02,\n",
       "            2.99893349e-01],\n",
       "          [-2.29371905e-01,  2.14777499e-01, -1.09476149e-01,\n",
       "           -2.26079524e-02]]],\n",
       " \n",
       " \n",
       "        [[[-1.54422626e-01,  1.15456223e-01, -3.07255179e-01,\n",
       "           -8.39172155e-02],\n",
       "          [-1.65100649e-01,  2.58447438e-01, -1.76642463e-01,\n",
       "           -6.11158907e-02],\n",
       "          [ 1.49723530e-01, -2.77481914e-01,  1.34591937e-01,\n",
       "            1.18613511e-01]],\n",
       " \n",
       "         [[-1.63108170e-01, -2.56819129e-01,  2.45335788e-01,\n",
       "            9.94941592e-02],\n",
       "          [ 1.23392761e-01, -2.16071427e-01,  2.03511983e-01,\n",
       "           -2.80239910e-01],\n",
       "          [ 2.10148424e-01,  1.08045727e-01,  2.32045621e-01,\n",
       "           -7.20775425e-02]],\n",
       " \n",
       "         [[-2.48062313e-02,  2.02733248e-01, -7.28464276e-02,\n",
       "           -7.71734118e-03],\n",
       "          [-2.69965976e-01,  6.41948879e-02,  2.93880701e-04,\n",
       "           -8.53917748e-02],\n",
       "          [ 2.32962817e-01,  2.75042742e-01,  1.35035664e-01,\n",
       "            5.13741374e-02]]],\n",
       " \n",
       " \n",
       "        [[[ 7.55320787e-02,  2.49723881e-01,  1.41119361e-01,\n",
       "            4.39321399e-02],\n",
       "          [-2.32266784e-02,  2.10110635e-01,  1.61423534e-01,\n",
       "           -2.09805429e-01],\n",
       "          [-4.13600802e-02, -2.13544190e-01, -2.90043741e-01,\n",
       "            3.07557881e-02]],\n",
       " \n",
       "         [[ 1.09604895e-02,  1.74953938e-01, -2.46319979e-01,\n",
       "           -1.05993494e-01],\n",
       "          [-1.72242820e-01,  2.91337460e-01, -1.41701132e-01,\n",
       "            1.97743207e-01],\n",
       "          [ 2.67041773e-01, -2.63941079e-01, -3.87778878e-02,\n",
       "           -7.58503079e-02]],\n",
       " \n",
       "         [[-2.76945978e-01, -8.47554207e-03, -2.16774255e-01,\n",
       "           -2.14015976e-01],\n",
       "          [ 2.16670126e-01,  2.44480640e-01, -1.07056692e-01,\n",
       "           -3.11981440e-02],\n",
       "          [ 1.72147244e-01, -1.05475947e-01,  1.62134767e-02,\n",
       "           -7.02604800e-02]]]], dtype=float32)>,\n",
       " <tf.Variable 'conv2d/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 返回所有待优化张量列表\n",
    "layer.trainable_variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 卷积层变种\n",
    "\n",
    "### 空洞卷积"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([1, 3, 3, 1])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.random.normal([1,7,7,1]) # 模拟输入\n",
    "# 空洞卷积， 1 个 3x3 的卷积核\n",
    "layer = layers.Conv2D(1,kernel_size=3,strides=1,dilation_rate=2)\n",
    "out = layer(x) # 前向计算\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 转置卷积"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=262, shape=(1, 2, 2, 1), dtype=float32, numpy=\n",
       "array([[[[ -67.],\n",
       "         [ -77.]],\n",
       "\n",
       "        [[-117.],\n",
       "         [-127.]]]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 创建 X 矩阵，高宽为 5x5\n",
    "x = tf.range(25)+1\n",
    "# Reshape 为合法维度的张量\n",
    "x = tf.reshape(x,[1,5,5,1])\n",
    "x = tf.cast(x, tf.float32)\n",
    "# 创建固定内容的卷积核矩阵\n",
    "w = tf.constant([[-1,2,-3.],[4,-5,6],[-7,8,-9]])\n",
    "# 调整为合法维度的张量\n",
    "w = tf.expand_dims(w,axis=2)\n",
    "w = tf.expand_dims(w,axis=3)\n",
    "# 进行普通卷积运算\n",
    "out = tf.nn.conv2d(x,w,strides=2,padding='VALID')\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=264, shape=(1, 5, 5, 1), dtype=float32, numpy=\n",
       "array([[[[   67.],\n",
       "         [ -134.],\n",
       "         [  278.],\n",
       "         [ -154.],\n",
       "         [  231.]],\n",
       "\n",
       "        [[ -268.],\n",
       "         [  335.],\n",
       "         [ -710.],\n",
       "         [  385.],\n",
       "         [ -462.]],\n",
       "\n",
       "        [[  586.],\n",
       "         [ -770.],\n",
       "         [ 1620.],\n",
       "         [ -870.],\n",
       "         [ 1074.]],\n",
       "\n",
       "        [[ -468.],\n",
       "         [  585.],\n",
       "         [-1210.],\n",
       "         [  635.],\n",
       "         [ -762.]],\n",
       "\n",
       "        [[  819.],\n",
       "         [ -936.],\n",
       "         [ 1942.],\n",
       "         [-1016.],\n",
       "         [ 1143.]]]], dtype=float32)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 普通卷积的输出作为转置卷积的输入，进行转置卷积运算\n",
    "xx = tf.nn.conv2d_transpose(out, w, strides=2,\n",
    "    padding='VALID',\n",
    "    output_shape=[1,5,5,1])\n",
    "xx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([1, 2, 2, 1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.random.normal([1,6,6,1])\n",
    "# 6x6 的输入经过普通卷积\n",
    "out = tf.nn.conv2d(x,w,strides=2,padding='VALID')\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=273, shape=(1, 6, 6, 1), dtype=float32, numpy=\n",
       "array([[[[ -12.10025  ],\n",
       "         [  24.2005   ],\n",
       "         [ -41.14547  ],\n",
       "         [   9.689438 ],\n",
       "         [ -14.534157 ],\n",
       "         [   0.       ]],\n",
       "\n",
       "        [[  48.401    ],\n",
       "         [ -60.50125  ],\n",
       "         [  91.98038  ],\n",
       "         [ -24.223595 ],\n",
       "         [  29.068314 ],\n",
       "         [   0.       ]],\n",
       "\n",
       "        [[-108.83326  ],\n",
       "         [ 145.06502  ],\n",
       "         [-199.14264  ],\n",
       "         [   6.6234093],\n",
       "         [   4.599045 ],\n",
       "         [   0.       ]],\n",
       "\n",
       "        [[  96.526024 ],\n",
       "         [-120.65753  ],\n",
       "         [  80.52035  ],\n",
       "         [  80.33585  ],\n",
       "         [ -96.40303  ],\n",
       "         [   0.       ]],\n",
       "\n",
       "        [[-168.92055  ],\n",
       "         [ 193.05205  ],\n",
       "         [-104.71335  ],\n",
       "         [-128.53737  ],\n",
       "         [ 144.60454  ],\n",
       "         [   0.       ]],\n",
       "\n",
       "        [[   0.       ],\n",
       "         [   0.       ],\n",
       "         [   0.       ],\n",
       "         [   0.       ],\n",
       "         [   0.       ],\n",
       "         [   0.       ]]]], dtype=float32)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 恢复出 6x6 大小\n",
    "xx = tf.nn.conv2d_transpose(out, w, strides=2,\n",
    "    padding='VALID',\n",
    "    output_shape=[1,6,6,1])\n",
    "xx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**转置卷积实现**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=288, shape=(1, 2, 2, 1), dtype=float32, numpy=\n",
       "array([[[[-56.],\n",
       "         [-61.]],\n",
       "\n",
       "        [[-76.],\n",
       "         [-81.]]]], dtype=float32)>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 创建 4x4 大小的输入\n",
    "x = tf.range(16)+1\n",
    "x = tf.reshape(x,[1,4,4,1])\n",
    "x = tf.cast(x, tf.float32)\n",
    "# 创建 3x3 卷积核\n",
    "w = tf.constant([[-1,2,-3.],[4,-5,6],[-7,8,-9]])\n",
    "w = tf.expand_dims(w,axis=2)\n",
    "w = tf.expand_dims(w,axis=3)\n",
    "# 普通卷积运算\n",
    "out = tf.nn.conv2d(x,w,strides=1,padding='VALID')\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在保持strides=1， padding='VALID'，卷积核不变的情况下，我们通过卷积核w与输出out的转置卷积运算尝试恢复与输入x相同大小的高宽张量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=291, shape=(4, 4), dtype=float32, numpy=\n",
       "array([[  56.,  -51.,   46.,  183.],\n",
       "       [-148.,  -35.,   35., -123.],\n",
       "       [  88.,   35.,  -35.,   63.],\n",
       "       [ 532.,  -41.,   36.,  729.]], dtype=float32)>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 恢复 4x4 大小的输入\n",
    "xx = tf.nn.conv2d_transpose(out, w, strides=1, padding='VALID',\n",
    "    output_shape=[1,4,4,1])\n",
    "tf.squeeze(xx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=344, shape=(1, 4, 4, 1), dtype=float32, numpy=\n",
       "array([[[[  1.896018  ],\n",
       "         [ 26.08768   ],\n",
       "         [ 57.787025  ],\n",
       "         [ 34.44299   ]],\n",
       "\n",
       "        [[ 24.278782  ],\n",
       "         [ 89.65541   ],\n",
       "         [ 82.10664   ],\n",
       "         [ 14.191902  ]],\n",
       "\n",
       "        [[  1.3896465 ],\n",
       "         [ 47.111137  ],\n",
       "         [ -0.22128582],\n",
       "         [-53.176964  ]],\n",
       "\n",
       "        [[-38.092247  ],\n",
       "         [-34.26138   ],\n",
       "         [ -7.313441  ],\n",
       "         [-14.992759  ]]]], dtype=float32)>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 创建转置卷积类\n",
    "layer = layers.Conv2DTranspose(1,kernel_size=3,strides=1,padding='VALID')\n",
    "xx2 = layer(out) # 通过转置卷积层\n",
    "xx2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  分离卷积"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "那么采用分离卷积有什么优势呢？一个很明显的优势在于， 同样的输入和输出，采用Separable Convolution 的参数量约是普通卷积的$\\displaystyle \\frac{1}{3}$。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 深度残差网络\n",
    "\n",
    "### ResBlock实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicBlock(layers.Layer):\n",
    "    # 残差模块类\n",
    "    def __init__(self, filter_num, stride=1):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        # f(x)包含了 2 个普通卷积层，创建卷积层 1\n",
    "        self.conv1 = layers.Conv2D(filter_num, (3, 3), strides=stride, padding='same')\n",
    "        self.bn1 = layers.BatchNormalization()\n",
    "        self.relu = layers.Activation('relu')\n",
    "        # 创建卷积层 2\n",
    "        self.conv2 = layers.Conv2D(filter_num, (3, 3), strides=1, padding='same')\n",
    "        self.bn2 = layers.BatchNormalization()\n",
    "    \n",
    "        if stride != 1: # 插入 identity 层\n",
    "            self.downsample = Sequential()\n",
    "            self.downsample.add(layers.Conv2D(filter_num, (1, 1), strides=stride))\n",
    "        else: # 否则，直接连接\n",
    "            self.downsample = lambda x:x\n",
    "            \n",
    "    def call(self, inputs, training=None):\n",
    "        # 前向传播函数\n",
    "        out = self.conv1(inputs) # 通过第一个卷积层\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.conv2(out) # 通过第二个卷积层\n",
    "        out = self.bn2(out)\n",
    "        # 输入通过 identity()转换\n",
    "        identity = self.downsample(inputs)\n",
    "        # f(x)+x 运算\n",
    "        output = layers.add([out, identity])\n",
    "        # 再通过激活函数并返回\n",
    "        output = tf.nn.relu(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
